{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "train.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ktbQAC-Xfvku",
        "colab_type": "text"
      },
      "source": [
        "## Data Directory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "id": "FQ0GwLD7fkO3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "print(os.listdir(\"../input\"))\n",
        "print(os.listdir(\"../input/dataset/dataset\"))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rtcL6uLDfkO_",
        "colab_type": "text"
      },
      "source": [
        "## Install Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BtFHREU2fkPA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "print(torch.__version__)\n",
        "print(torch.cuda.device_count())\n",
        "print(torch.cuda.is_available())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hPjrVskrfkPD",
        "colab_type": "text"
      },
      "source": [
        "## Import libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KBoTqWBlfkPE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import cv2\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from torch.utils import data\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.optim.lr_scheduler import MultiStepLR"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3y6zGj92fkPG",
        "colab_type": "text"
      },
      "source": [
        "## Hyper-parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDOKqTl6fkPH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataroot = \"../input/dataset/dataset/\"\n",
        "ckptroot = \"./\"\n",
        "\n",
        "lr = 1e-4\n",
        "weight_decay = 1e-5\n",
        "batch_size = 32\n",
        "num_workers = 8\n",
        "test_size = 0.8\n",
        "shuffle = True\n",
        "\n",
        "epochs = 80\n",
        "start_epoch = 0\n",
        "resume = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U7E4M2ynfkPJ",
        "colab_type": "text"
      },
      "source": [
        "## Helper functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "swFFOrkRfkPK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def toDevice(datas, device):\n",
        "    \"\"\"Enable cuda.\"\"\"\n",
        "    imgs, angles = datas\n",
        "    return imgs.float().to(device), angles.float().to(device)\n",
        "\n",
        "\n",
        "def augment(dataroot, imgName, angle):\n",
        "    \"\"\"Data augmentation.\"\"\"\n",
        "    name = dataroot + 'IMG/' + imgName.split('\\\\')[-1]\n",
        "    current_image = cv2.imread(name)\n",
        "\n",
        "    if current_image is None:\n",
        "        print(name)\n",
        "\n",
        "    current_image = current_image[65:-25, :, :]\n",
        "    if np.random.rand() < 0.5:\n",
        "        current_image = cv2.flip(current_image, 1)\n",
        "        angle = angle * -1.0\n",
        "\n",
        "    return current_image, angle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yOo86RC1fkPM",
        "colab_type": "text"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j0n38m6vfkPN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import scipy\n",
        "from scipy import signal\n",
        "\n",
        "def load_data(data_dir, test_size):\n",
        "    \"\"\"Load training data and train validation split\"\"\"\n",
        "    # reads CSV file into a single dataframe variable\n",
        "    data_df = pd.read_csv(os.path.join(data_dir, 'driving_log.csv'),\n",
        "                          names=['center', 'left', 'right', 'steering', 'throttle', 'reverse', 'speed'])\n",
        "\n",
        "    # smooth data signal with `savgol_filter`\n",
        "    data_df[\"steering\"] = signal.savgol_filter(data_df[\"steering\"].values.tolist(), 51, 11)\n",
        "\n",
        "    # Divide the data into training set and validation set\n",
        "    train_len = int(test_size * data_df.shape[0])\n",
        "    valid_len = data_df.shape[0] - train_len\n",
        "    trainset, valset = data.random_split(\n",
        "        data_df.values.tolist(), lengths=[train_len, valid_len])\n",
        "\n",
        "    return trainset, valset\n",
        "\n",
        "trainset, valset = load_data(dataroot, test_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BzY3zIFdfkPb",
        "colab_type": "text"
      },
      "source": [
        "## Create dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DpSjiP-fkPf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class TripletDataset(data.Dataset):\n",
        "\n",
        "    def __init__(self, dataroot, samples, transform=None):\n",
        "        self.samples = samples\n",
        "        self.dataroot = dataroot\n",
        "        self.transform = transform\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        batch_samples = self.samples[index]\n",
        "        steering_angle = float(batch_samples[3])\n",
        "\n",
        "        center_img, steering_angle_center = augment(self.dataroot, batch_samples[0], steering_angle)\n",
        "        left_img, steering_angle_left     = augment(self.dataroot, batch_samples[1], steering_angle + 0.4)\n",
        "        right_img, steering_angle_right   = augment(self.dataroot, batch_samples[2], steering_angle - 0.4)\n",
        "\n",
        "        center_img = self.transform(center_img)\n",
        "        left_img   = self.transform(left_img)\n",
        "        right_img  = self.transform(right_img)\n",
        "\n",
        "        return (center_img, steering_angle_center), (left_img, steering_angle_left), (right_img, steering_angle_right)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.samples)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FUfByxjNfkPj",
        "colab_type": "text"
      },
      "source": [
        "## Get data loader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GV_h604FfkPk",
        "colab_type": "code",
        "outputId": "ebdd8196-2093-44ca-a56c-5465f4b816d6",
        "colab": {}
      },
      "source": [
        "print(\"==> Preparing dataset ...\")\n",
        "def data_loader(dataroot, trainset, valset, batch_size, shuffle, num_workers):\n",
        "    \"\"\"Self-Driving vehicles simulator dataset Loader.\n",
        "\n",
        "    Args:\n",
        "        trainset: training set\n",
        "        valset: validation set\n",
        "        batch_size: training set input batch size\n",
        "        shuffle: whether shuffle during training process\n",
        "        num_workers: number of workers in DataLoader\n",
        "\n",
        "    Returns:\n",
        "        trainloader (torch.utils.data.DataLoader): DataLoader for training set\n",
        "        testloader (torch.utils.data.DataLoader): DataLoader for validation set\n",
        "    \"\"\"\n",
        "    transformations = transforms.Compose(\n",
        "        [transforms.Lambda(lambda x: (x / 127.5) - 1.0)])\n",
        "\n",
        "    # Load training data and validation data\n",
        "    training_set = TripletDataset(dataroot, trainset, transformations)\n",
        "    trainloader = DataLoader(training_set,\n",
        "                             batch_size=batch_size,\n",
        "                             shuffle=shuffle,\n",
        "                             num_workers=num_workers)\n",
        "\n",
        "    validation_set = TripletDataset(dataroot, valset, transformations)\n",
        "    valloader = DataLoader(validation_set,\n",
        "                           batch_size=batch_size,\n",
        "                           shuffle=shuffle,\n",
        "                           num_workers=num_workers)\n",
        "\n",
        "    return trainloader, valloader\n",
        "\n",
        "\n",
        "trainloader, validationloader = data_loader(dataroot,\n",
        "                                            trainset, valset,\n",
        "                                            batch_size,\n",
        "                                            shuffle,\n",
        "                                            num_workers)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "==> Preparing dataset ...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dDdWHeMifkPn",
        "colab_type": "text"
      },
      "source": [
        "## Define model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1K-KmFHWfkPo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class NetworkNvidia(nn.Module):\n",
        "    \"\"\"NVIDIA model used in the paper.\"\"\"\n",
        "\n",
        "    def __init__(self):\n",
        "        \"\"\"Initialize NVIDIA model.\n",
        "\n",
        "        NVIDIA model used\n",
        "            Image normalization to avoid saturation and make gradients work better.\n",
        "            Convolution: 5x5, filter: 24, strides: 2x2, activation: ELU\n",
        "            Convolution: 5x5, filter: 36, strides: 2x2, activation: ELU\n",
        "            Convolution: 5x5, filter: 48, strides: 2x2, activation: ELU\n",
        "            Convolution: 3x3, filter: 64, strides: 1x1, activation: ELU\n",
        "            Convolution: 3x3, filter: 64, strides: 1x1, activation: ELU\n",
        "            Drop out (0.5)\n",
        "            Fully connected: neurons: 100, activation: ELU\n",
        "            Fully connected: neurons: 50, activation: ELU\n",
        "            Fully connected: neurons: 10, activation: ELU\n",
        "            Fully connected: neurons: 1 (output)\n",
        "\n",
        "        the convolution layers are meant to handle feature engineering\n",
        "        the fully connected layer for predicting the steering angle.\n",
        "        \"\"\"\n",
        "        super(NetworkNvidia, self).__init__()\n",
        "        self.conv_layers = nn.Sequential(\n",
        "            nn.Conv2d(3, 24, 5, stride=2),\n",
        "            nn.ELU(),\n",
        "            nn.Conv2d(24, 36, 5, stride=2),\n",
        "            nn.ELU(),\n",
        "            nn.Conv2d(36, 48, 5, stride=2),\n",
        "            nn.ELU(),\n",
        "            nn.Conv2d(48, 64, 3),\n",
        "            nn.ELU(),\n",
        "            nn.Conv2d(64, 64, 3),\n",
        "            nn.Dropout(0.5)\n",
        "        )\n",
        "        self.linear_layers = nn.Sequential(\n",
        "            nn.Linear(in_features=64 * 2 * 33, out_features=100),\n",
        "            nn.ELU(),\n",
        "            nn.Linear(in_features=100, out_features=50),\n",
        "            nn.ELU(),\n",
        "            nn.Linear(in_features=50, out_features=10),\n",
        "            nn.Linear(in_features=10, out_features=1)\n",
        "        )\n",
        "\n",
        "    def forward(self, input):\n",
        "        \"\"\"Forward pass.\"\"\"\n",
        "        input = input.view(input.size(0), 3, 70, 320)\n",
        "        output = self.conv_layers(input)\n",
        "        # print(output.shape)\n",
        "        output = output.view(output.size(0), -1)\n",
        "        output = self.linear_layers(output)\n",
        "        return output\n",
        "\n",
        "\n",
        "# Define model\n",
        "print(\"==> Initialize model ...\")\n",
        "model = NetworkNvidia()\n",
        "print(\"==> Initialize model done ...\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HxGAzAzDfkPu",
        "colab_type": "text"
      },
      "source": [
        "## Define optimizer and criterion"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_iH3OhLBfkPu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Define optimizer and criterion\n",
        "optimizer = optim.Adam(model.parameters(),\n",
        "                       lr=lr,\n",
        "                       weight_decay=weight_decay)\n",
        "criterion = nn.MSELoss()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pw1poQoYfkPy",
        "colab_type": "text"
      },
      "source": [
        "## Learning rate scheduler"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P1MCyJhSfkPz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# learning rate scheduler\n",
        "scheduler = MultiStepLR(optimizer, milestones=[30, 50], gamma=0.1)\n",
        "\n",
        "# transfer to gpu\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OM520SyQfkP1",
        "colab_type": "text"
      },
      "source": [
        "## Resume training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "luVRBv3CfkP2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if resume:\n",
        "    print(\"==> Loading checkpoint ...\")\n",
        "    checkpoint = torch.load(\"../input/pretrainedmodels/both-nvidia-model-61.h5\",\n",
        "                            map_location=lambda storage, loc: storage)\n",
        "    start_epoch = checkpoint['epoch']\n",
        "    model.load_state_dict(checkpoint['state_dict'])\n",
        "    optimizer.load_state_dict(checkpoint['optimizer'])\n",
        "    scheduler.load_state_dict(checkpoint['scheduler'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3f1RloYefkP3",
        "colab_type": "text"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gC9EK0w0fkP5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Trainer(object):\n",
        "    \"\"\"Trainer.\"\"\"\n",
        "\n",
        "    def __init__(self,\n",
        "                 ckptroot,\n",
        "                 model,\n",
        "                 device,\n",
        "                 epochs,\n",
        "                 criterion,\n",
        "                 optimizer,\n",
        "                 scheduler,\n",
        "                 start_epoch,\n",
        "                 trainloader,\n",
        "                 validationloader):\n",
        "        \"\"\"Self-Driving car Trainer.\n",
        "\n",
        "        Args:\n",
        "            model:\n",
        "            device:\n",
        "            epochs:\n",
        "            criterion:\n",
        "            optimizer:\n",
        "            start_epoch:\n",
        "            trainloader:\n",
        "            validationloader:\n",
        "\n",
        "        \"\"\"\n",
        "        super(Trainer, self).__init__()\n",
        "\n",
        "        self.model = model\n",
        "        self.device = device\n",
        "        self.epochs = epochs\n",
        "        self.ckptroot = ckptroot\n",
        "        self.criterion = criterion\n",
        "        self.optimizer = optimizer\n",
        "        self.scheduler = scheduler\n",
        "        self.start_epoch = start_epoch\n",
        "        self.trainloader = trainloader\n",
        "        self.validationloader = validationloader\n",
        "\n",
        "    def train(self):\n",
        "        \"\"\"Training process.\"\"\"\n",
        "        self.model.to(self.device)\n",
        "        for epoch in range(self.start_epoch, self.epochs + self.start_epoch):\n",
        "            self.scheduler.step()\n",
        "            \n",
        "            # Training\n",
        "            train_loss = 0.0\n",
        "            self.model.train()\n",
        "\n",
        "            for local_batch, (centers, lefts, rights) in enumerate(self.trainloader):\n",
        "                # Transfer to GPU\n",
        "                centers, lefts, rights = toDevice(centers, self.device), toDevice(\n",
        "                    lefts, self.device), toDevice(rights, self.device)\n",
        "\n",
        "                # Model computations\n",
        "                self.optimizer.zero_grad()\n",
        "                datas = [centers, lefts, rights]\n",
        "                for data in datas:\n",
        "                    imgs, angles = data\n",
        "                    # print(\"training image: \", imgs.shape)\n",
        "                    outputs = self.model(imgs)\n",
        "                    loss = self.criterion(outputs, angles.unsqueeze(1))\n",
        "                    loss.backward()\n",
        "                    self.optimizer.step()\n",
        "\n",
        "                    train_loss += loss.data.item()\n",
        "\n",
        "                if local_batch % 100 == 0:\n",
        "\n",
        "                    print(\"Training Epoch: {} | Loss: {}\".format(epoch, train_loss / (local_batch + 1)))\n",
        "\n",
        "            # Validation\n",
        "            self.model.eval()\n",
        "            valid_loss = 0\n",
        "            with torch.set_grad_enabled(False):\n",
        "                for local_batch, (centers, lefts, rights) in enumerate(self.validationloader):\n",
        "                    # Transfer to GPU\n",
        "                    centers, lefts, rights = toDevice(centers, self.device), toDevice(\n",
        "                        lefts, self.device), toDevice(rights, self.device)\n",
        "\n",
        "                    # Model computations\n",
        "                    self.optimizer.zero_grad()\n",
        "                    datas = [centers, lefts, rights]\n",
        "                    for data in datas:\n",
        "                        imgs, angles = data\n",
        "                        outputs = self.model(imgs)\n",
        "                        loss = self.criterion(outputs, angles.unsqueeze(1))\n",
        "\n",
        "                        valid_loss += loss.data.item()\n",
        "\n",
        "                    if local_batch % 100 == 0:\n",
        "                        print(\"Validation Loss: {}\".format(valid_loss / (local_batch + 1)))\n",
        "\n",
        "            print()\n",
        "            # Save model\n",
        "            if epoch % 5 == 0 or epoch == self.epochs + self.start_epoch - 1:\n",
        "\n",
        "                state = {\n",
        "                    'epoch': epoch + 1,\n",
        "                    'state_dict': self.model.state_dict(),\n",
        "                    'optimizer': self.optimizer.state_dict(),\n",
        "                    'scheduler': self.scheduler.state_dict(),\n",
        "                }\n",
        "\n",
        "                self.save_checkpoint(state)\n",
        "\n",
        "    def save_checkpoint(self, state):\n",
        "        \"\"\"Save checkpoint.\"\"\"\n",
        "        print(\"==> Save checkpoint ...\")\n",
        "        if not os.path.exists(self.ckptroot):\n",
        "            os.makedirs(self.ckptroot)\n",
        "\n",
        "        torch.save(state, self.ckptroot + 'both-nvidia-model-{}.h5'.format(state['epoch']))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ycy2rSL3fkP7",
        "colab_type": "code",
        "outputId": "3f4bf9c9-bfca-4e2e-8810-1c47af381175",
        "colab": {}
      },
      "source": [
        "print(\"==> Start training ...\")\n",
        "trainer = Trainer(ckptroot,\n",
        "                  model,\n",
        "                  device,\n",
        "                  epochs,\n",
        "                  criterion,\n",
        "                  optimizer,\n",
        "                  scheduler,\n",
        "                  start_epoch,\n",
        "                  trainloader,\n",
        "                  validationloader)\n",
        "trainer.train()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "==> Start training ...\n",
            "Training Epoch: 0 | Loss: 0.5558980479836464\n",
            "Training Epoch: 0 | Loss: 0.5213123413337635\n",
            "Training Epoch: 0 | Loss: 0.49653078340438767\n",
            "Training Epoch: 0 | Loss: 0.47576030551075343\n",
            "Training Epoch: 0 | Loss: 0.46251533310070747\n",
            "Training Epoch: 0 | Loss: 0.45327764451912717\n",
            "Training Epoch: 0 | Loss: 0.4468134562632134\n",
            "Training Epoch: 0 | Loss: 0.43905620173167187\n",
            "Training Epoch: 0 | Loss: 0.4328681884645095\n",
            "Validation Loss: 0.3594933710992336\n",
            "Validation Loss: 0.3774320130780487\n",
            "Validation Loss: 0.3786888392688475\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 1 | Loss: 0.29850859567523\n",
            "Training Epoch: 1 | Loss: 0.3933299669424201\n",
            "Training Epoch: 1 | Loss: 0.38579951670595364\n",
            "Training Epoch: 1 | Loss: 0.3791089756445235\n",
            "Training Epoch: 1 | Loss: 0.37866771645249425\n",
            "Training Epoch: 1 | Loss: 0.3755420975595296\n",
            "Training Epoch: 1 | Loss: 0.37281983622509407\n",
            "Training Epoch: 1 | Loss: 0.37209821650266817\n",
            "Training Epoch: 1 | Loss: 0.37061068274722564\n",
            "Validation Loss: 0.4482746571302414\n",
            "Validation Loss: 0.36231463190426333\n",
            "Validation Loss: 0.35759038125067505\n",
            "\n",
            "Training Epoch: 2 | Loss: 0.4340990409255028\n",
            "Training Epoch: 2 | Loss: 0.3613892498767317\n",
            "Training Epoch: 2 | Loss: 0.3549202584675444\n",
            "Training Epoch: 2 | Loss: 0.35272254028738137\n",
            "Training Epoch: 2 | Loss: 0.35017852030434365\n",
            "Training Epoch: 2 | Loss: 0.3463931422821895\n",
            "Training Epoch: 2 | Loss: 0.3460471120270933\n",
            "Training Epoch: 2 | Loss: 0.34563853538538014\n",
            "Training Epoch: 2 | Loss: 0.3454248610367712\n",
            "Validation Loss: 0.3542107194662094\n",
            "Validation Loss: 0.3406148337676088\n",
            "Validation Loss: 0.33789137188363727\n",
            "\n",
            "Training Epoch: 3 | Loss: 0.264209995046258\n",
            "Training Epoch: 3 | Loss: 0.3261413291410202\n",
            "Training Epoch: 3 | Loss: 0.3273491282433049\n",
            "Training Epoch: 3 | Loss: 0.3265615521630031\n",
            "Training Epoch: 3 | Loss: 0.3258087210045974\n",
            "Training Epoch: 3 | Loss: 0.3259306916253206\n",
            "Training Epoch: 3 | Loss: 0.3268608287822908\n",
            "Training Epoch: 3 | Loss: 0.3264079918000799\n",
            "Training Epoch: 3 | Loss: 0.3259248299007949\n",
            "Validation Loss: 0.39915426820516586\n",
            "Validation Loss: 0.30457275844003895\n",
            "Validation Loss: 0.3131801426151202\n",
            "\n",
            "Training Epoch: 4 | Loss: 0.39218131452798843\n",
            "Training Epoch: 4 | Loss: 0.3071626303533074\n",
            "Training Epoch: 4 | Loss: 0.31109592095666116\n",
            "Training Epoch: 4 | Loss: 0.31579079903179524\n",
            "Training Epoch: 4 | Loss: 0.31195566433242655\n",
            "Training Epoch: 4 | Loss: 0.30986880071780937\n",
            "Training Epoch: 4 | Loss: 0.31034917343011165\n",
            "Training Epoch: 4 | Loss: 0.3110801278920406\n",
            "Training Epoch: 4 | Loss: 0.3119584239668186\n",
            "Validation Loss: 0.33516016602516174\n",
            "Validation Loss: 0.3022027490383918\n",
            "Validation Loss: 0.30148412708878813\n",
            "\n",
            "Training Epoch: 5 | Loss: 0.27030542120337486\n",
            "Training Epoch: 5 | Loss: 0.30442704625501493\n",
            "Training Epoch: 5 | Loss: 0.29826431486996546\n",
            "Training Epoch: 5 | Loss: 0.2994403130377943\n",
            "Training Epoch: 5 | Loss: 0.2985142806540255\n",
            "Training Epoch: 5 | Loss: 0.2981301402550436\n",
            "Training Epoch: 5 | Loss: 0.297343739291371\n",
            "Training Epoch: 5 | Loss: 0.2987090627371882\n",
            "Training Epoch: 5 | Loss: 0.2987920840790544\n",
            "Validation Loss: 0.32323939725756645\n",
            "Validation Loss: 0.29273371717496083\n",
            "Validation Loss: 0.2871951352879034\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 6 | Loss: 0.32839538902044296\n",
            "Training Epoch: 6 | Loss: 0.29666612228138906\n",
            "Training Epoch: 6 | Loss: 0.2934820310569447\n",
            "Training Epoch: 6 | Loss: 0.29186512748602517\n",
            "Training Epoch: 6 | Loss: 0.29133500919405586\n",
            "Training Epoch: 6 | Loss: 0.2883545456402108\n",
            "Training Epoch: 6 | Loss: 0.2876144118015089\n",
            "Training Epoch: 6 | Loss: 0.28617462163894686\n",
            "Training Epoch: 6 | Loss: 0.2859632329801365\n",
            "Validation Loss: 0.3766295313835144\n",
            "Validation Loss: 0.2782088677113009\n",
            "Validation Loss: 0.2822243148155177\n",
            "\n",
            "Training Epoch: 7 | Loss: 0.2903166115283966\n",
            "Training Epoch: 7 | Loss: 0.2865490806611753\n",
            "Training Epoch: 7 | Loss: 0.28307681777213345\n",
            "Training Epoch: 7 | Loss: 0.28187749710589943\n",
            "Training Epoch: 7 | Loss: 0.2844150026242929\n",
            "Training Epoch: 7 | Loss: 0.28225355822511183\n",
            "Training Epoch: 7 | Loss: 0.2826199128808525\n",
            "Training Epoch: 7 | Loss: 0.2808552694330244\n",
            "Training Epoch: 7 | Loss: 0.27950089134754946\n",
            "Validation Loss: 0.2773801423609257\n",
            "Validation Loss: 0.2668501916203168\n",
            "Validation Loss: 0.2679719563471663\n",
            "\n",
            "Training Epoch: 8 | Loss: 0.22995337285101414\n",
            "Training Epoch: 8 | Loss: 0.2671015135538165\n",
            "Training Epoch: 8 | Loss: 0.2693308888012497\n",
            "Training Epoch: 8 | Loss: 0.2699474113253858\n",
            "Training Epoch: 8 | Loss: 0.2702809993474932\n",
            "Training Epoch: 8 | Loss: 0.27097151106494866\n",
            "Training Epoch: 8 | Loss: 0.269372782569236\n",
            "Training Epoch: 8 | Loss: 0.2703229691275261\n",
            "Training Epoch: 8 | Loss: 0.2694994483233782\n",
            "Validation Loss: 0.4028911516070366\n",
            "Validation Loss: 0.26184657857854765\n",
            "Validation Loss: 0.2637266797935636\n",
            "\n",
            "Training Epoch: 9 | Loss: 0.21294931136071682\n",
            "Training Epoch: 9 | Loss: 0.2616017740404252\n",
            "Training Epoch: 9 | Loss: 0.26171125053655153\n",
            "Training Epoch: 9 | Loss: 0.26092398835599817\n",
            "Training Epoch: 9 | Loss: 0.2617953882373851\n",
            "Training Epoch: 9 | Loss: 0.26141142010681345\n",
            "Training Epoch: 9 | Loss: 0.2625510316208848\n",
            "Training Epoch: 9 | Loss: 0.2610523397549349\n",
            "Training Epoch: 9 | Loss: 0.2612313608651732\n",
            "Validation Loss: 0.20352470874786377\n",
            "Validation Loss: 0.2572043027157093\n",
            "Validation Loss: 0.2567003251410168\n",
            "\n",
            "Training Epoch: 10 | Loss: 0.21185582876205444\n",
            "Training Epoch: 10 | Loss: 0.2526186974073696\n",
            "Training Epoch: 10 | Loss: 0.25129839042618646\n",
            "Training Epoch: 10 | Loss: 0.25301170805500095\n",
            "Training Epoch: 10 | Loss: 0.2529331938505284\n",
            "Training Epoch: 10 | Loss: 0.2524757189341559\n",
            "Training Epoch: 10 | Loss: 0.2522906418982738\n",
            "Training Epoch: 10 | Loss: 0.2547447567888019\n",
            "Training Epoch: 10 | Loss: 0.25426639573353244\n",
            "Validation Loss: 0.17803704738616943\n",
            "Validation Loss: 0.24933554828720222\n",
            "Validation Loss: 0.25088364510004646\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 11 | Loss: 0.28222727589309216\n",
            "Training Epoch: 11 | Loss: 0.24758459486546788\n",
            "Training Epoch: 11 | Loss: 0.24390571605210282\n",
            "Training Epoch: 11 | Loss: 0.2457734851531709\n",
            "Training Epoch: 11 | Loss: 0.24518923855333258\n",
            "Training Epoch: 11 | Loss: 0.24370857883623975\n",
            "Training Epoch: 11 | Loss: 0.24585306418814795\n",
            "Training Epoch: 11 | Loss: 0.24607789625922555\n",
            "Training Epoch: 11 | Loss: 0.24603650066420232\n",
            "Validation Loss: 0.29393066465854645\n",
            "Validation Loss: 0.24517780712040343\n",
            "Validation Loss: 0.24816482426107522\n",
            "\n",
            "Training Epoch: 12 | Loss: 0.23766812682151794\n",
            "Training Epoch: 12 | Loss: 0.24573293279413835\n",
            "Training Epoch: 12 | Loss: 0.2435140684154348\n",
            "Training Epoch: 12 | Loss: 0.24455862184769886\n",
            "Training Epoch: 12 | Loss: 0.24235888323433083\n",
            "Training Epoch: 12 | Loss: 0.24117271392280887\n",
            "Training Epoch: 12 | Loss: 0.24186863172985304\n",
            "Training Epoch: 12 | Loss: 0.24082272260259843\n",
            "Training Epoch: 12 | Loss: 0.24107306878130125\n",
            "Validation Loss: 0.24100221879780293\n",
            "Validation Loss: 0.24739118067934962\n",
            "Validation Loss: 0.2439304100377346\n",
            "\n",
            "Training Epoch: 13 | Loss: 0.25456971675157547\n",
            "Training Epoch: 13 | Loss: 0.23461813835593143\n",
            "Training Epoch: 13 | Loss: 0.2338196463113771\n",
            "Training Epoch: 13 | Loss: 0.23472174196086354\n",
            "Training Epoch: 13 | Loss: 0.2332427940932927\n",
            "Training Epoch: 13 | Loss: 0.233152847989888\n",
            "Training Epoch: 13 | Loss: 0.23435691840630551\n",
            "Training Epoch: 13 | Loss: 0.23435870556163466\n",
            "Training Epoch: 13 | Loss: 0.2337003994172674\n",
            "Validation Loss: 0.17068983986973763\n",
            "Validation Loss: 0.23284707438790858\n",
            "Validation Loss: 0.23032822598476166\n",
            "\n",
            "Training Epoch: 14 | Loss: 0.20500125363469124\n",
            "Training Epoch: 14 | Loss: 0.23083460223999355\n",
            "Training Epoch: 14 | Loss: 0.22811559161903402\n",
            "Training Epoch: 14 | Loss: 0.2295534267705986\n",
            "Training Epoch: 14 | Loss: 0.22893911445666654\n",
            "Training Epoch: 14 | Loss: 0.2293677601624957\n",
            "Training Epoch: 14 | Loss: 0.2287822798864541\n",
            "Training Epoch: 14 | Loss: 0.22885720480297203\n",
            "Training Epoch: 14 | Loss: 0.22837581821413858\n",
            "Validation Loss: 0.281342051923275\n",
            "Validation Loss: 0.22756358607681376\n",
            "Validation Loss: 0.22592272661366866\n",
            "\n",
            "Training Epoch: 15 | Loss: 0.18625633418560028\n",
            "Training Epoch: 15 | Loss: 0.22155093053116068\n",
            "Training Epoch: 15 | Loss: 0.22841702257537871\n",
            "Training Epoch: 15 | Loss: 0.22528333675759873\n",
            "Training Epoch: 15 | Loss: 0.2237345713940902\n",
            "Training Epoch: 15 | Loss: 0.22327746673972307\n",
            "Training Epoch: 15 | Loss: 0.22320071807729325\n",
            "Training Epoch: 15 | Loss: 0.22208592686775577\n",
            "Training Epoch: 15 | Loss: 0.2224890992984864\n",
            "Validation Loss: 0.20852447301149368\n",
            "Validation Loss: 0.22743604202192313\n",
            "Validation Loss: 0.22756089310889221\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 16 | Loss: 0.278608076274395\n",
            "Training Epoch: 16 | Loss: 0.2176758324782742\n",
            "Training Epoch: 16 | Loss: 0.22105828319578918\n",
            "Training Epoch: 16 | Loss: 0.22038722468967056\n",
            "Training Epoch: 16 | Loss: 0.21867911999771125\n",
            "Training Epoch: 16 | Loss: 0.21758490169751726\n",
            "Training Epoch: 16 | Loss: 0.21735895485981382\n",
            "Training Epoch: 16 | Loss: 0.21838709287818378\n",
            "Training Epoch: 16 | Loss: 0.21816295476283845\n",
            "Validation Loss: 0.21124553307890892\n",
            "Validation Loss: 0.22580153291288874\n",
            "Validation Loss: 0.22567729530531672\n",
            "\n",
            "Training Epoch: 17 | Loss: 0.2221863493323326\n",
            "Training Epoch: 17 | Loss: 0.20700035888775445\n",
            "Training Epoch: 17 | Loss: 0.20867448877913886\n",
            "Training Epoch: 17 | Loss: 0.20947744138848248\n",
            "Training Epoch: 17 | Loss: 0.21001340445837102\n",
            "Training Epoch: 17 | Loss: 0.21080476700322714\n",
            "Training Epoch: 17 | Loss: 0.21220476588485343\n",
            "Training Epoch: 17 | Loss: 0.2127552761958006\n",
            "Training Epoch: 17 | Loss: 0.21344786353604625\n",
            "Validation Loss: 0.2768826875835657\n",
            "Validation Loss: 0.2207322043209973\n",
            "Validation Loss: 0.21826668909347768\n",
            "\n",
            "Training Epoch: 18 | Loss: 0.21707474067807198\n",
            "Training Epoch: 18 | Loss: 0.2100564966088917\n",
            "Training Epoch: 18 | Loss: 0.2095325038819319\n",
            "Training Epoch: 18 | Loss: 0.2067850856481745\n",
            "Training Epoch: 18 | Loss: 0.20531453758459584\n",
            "Training Epoch: 18 | Loss: 0.20554040704190196\n",
            "Training Epoch: 18 | Loss: 0.2064526474301683\n",
            "Training Epoch: 18 | Loss: 0.20697355608770707\n",
            "Training Epoch: 18 | Loss: 0.20740818699935776\n",
            "Validation Loss: 0.15573538094758987\n",
            "Validation Loss: 0.2093696068271552\n",
            "Validation Loss: 0.20964083010655138\n",
            "\n",
            "Training Epoch: 19 | Loss: 0.17430533096194267\n",
            "Training Epoch: 19 | Loss: 0.2051358608676508\n",
            "Training Epoch: 19 | Loss: 0.20169026392340586\n",
            "Training Epoch: 19 | Loss: 0.20207260390496085\n",
            "Training Epoch: 19 | Loss: 0.20188577775204755\n",
            "Training Epoch: 19 | Loss: 0.20216206715213592\n",
            "Training Epoch: 19 | Loss: 0.20159144903736162\n",
            "Training Epoch: 19 | Loss: 0.2022220491864765\n",
            "Training Epoch: 19 | Loss: 0.20246625956340916\n",
            "Validation Loss: 0.2369263619184494\n",
            "Validation Loss: 0.20266468283526673\n",
            "Validation Loss: 0.20726833708899384\n",
            "\n",
            "Training Epoch: 20 | Loss: 0.2222776673734188\n",
            "Training Epoch: 20 | Loss: 0.21027703304765838\n",
            "Training Epoch: 20 | Loss: 0.20322537491218515\n",
            "Training Epoch: 20 | Loss: 0.2017889144548843\n",
            "Training Epoch: 20 | Loss: 0.19995146813312373\n",
            "Training Epoch: 20 | Loss: 0.20040283582829488\n",
            "Training Epoch: 20 | Loss: 0.19980571134490102\n",
            "Training Epoch: 20 | Loss: 0.1993404284502232\n",
            "Training Epoch: 20 | Loss: 0.19946590290124647\n",
            "Validation Loss: 0.16440673358738422\n",
            "Validation Loss: 0.209560443158492\n",
            "Validation Loss: 0.20665648085672167\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 21 | Loss: 0.2762938588857651\n",
            "Training Epoch: 21 | Loss: 0.19057230524128616\n",
            "Training Epoch: 21 | Loss: 0.19143913665424978\n",
            "Training Epoch: 21 | Loss: 0.19268284435461328\n",
            "Training Epoch: 21 | Loss: 0.19391095649423148\n",
            "Training Epoch: 21 | Loss: 0.19397550788029522\n",
            "Training Epoch: 21 | Loss: 0.19316910364558168\n",
            "Training Epoch: 21 | Loss: 0.19361230589651346\n",
            "Training Epoch: 21 | Loss: 0.19401582251052918\n",
            "Validation Loss: 0.20578867197036743\n",
            "Validation Loss: 0.20170451565138478\n",
            "Validation Loss: 0.2017488433048129\n",
            "\n",
            "Training Epoch: 22 | Loss: 0.19112133979797363\n",
            "Training Epoch: 22 | Loss: 0.1864249930246779\n",
            "Training Epoch: 22 | Loss: 0.18730209411043136\n",
            "Training Epoch: 22 | Loss: 0.1895174391697659\n",
            "Training Epoch: 22 | Loss: 0.18932300684811024\n",
            "Training Epoch: 22 | Loss: 0.19022441885718508\n",
            "Training Epoch: 22 | Loss: 0.18973341082438158\n",
            "Training Epoch: 22 | Loss: 0.19124891420828974\n",
            "Training Epoch: 22 | Loss: 0.19155113258225828\n",
            "Validation Loss: 0.1396778579801321\n",
            "Validation Loss: 0.20069910235481686\n",
            "Validation Loss: 0.19718750514349534\n",
            "\n",
            "Training Epoch: 23 | Loss: 0.1680156160145998\n",
            "Training Epoch: 23 | Loss: 0.18342934296199\n",
            "Training Epoch: 23 | Loss: 0.1855131829880289\n",
            "Training Epoch: 23 | Loss: 0.18612864660253853\n",
            "Training Epoch: 23 | Loss: 0.18776062037563532\n",
            "Training Epoch: 23 | Loss: 0.18745538163669928\n",
            "Training Epoch: 23 | Loss: 0.18678500525771133\n",
            "Training Epoch: 23 | Loss: 0.18654204154793658\n",
            "Training Epoch: 23 | Loss: 0.1869311716661006\n",
            "Validation Loss: 0.2137889377772808\n",
            "Validation Loss: 0.1934741106677321\n",
            "Validation Loss: 0.20233425665276117\n",
            "\n",
            "Training Epoch: 24 | Loss: 0.17449133843183517\n",
            "Training Epoch: 24 | Loss: 0.18256820474594537\n",
            "Training Epoch: 24 | Loss: 0.18363206641313004\n",
            "Training Epoch: 24 | Loss: 0.18103405783738805\n",
            "Training Epoch: 24 | Loss: 0.18377421041973807\n",
            "Training Epoch: 24 | Loss: 0.18324544396206827\n",
            "Training Epoch: 24 | Loss: 0.18310828917751246\n",
            "Training Epoch: 24 | Loss: 0.1826808032886618\n",
            "Training Epoch: 24 | Loss: 0.18361921361025502\n",
            "Validation Loss: 0.21371613070368767\n",
            "Validation Loss: 0.19830838014546892\n",
            "Validation Loss: 0.19559160577346438\n",
            "\n",
            "Training Epoch: 25 | Loss: 0.1620171144604683\n",
            "Training Epoch: 25 | Loss: 0.17842964881496265\n",
            "Training Epoch: 25 | Loss: 0.17904694451121103\n",
            "Training Epoch: 25 | Loss: 0.17786293076753518\n",
            "Training Epoch: 25 | Loss: 0.1785723666156505\n",
            "Training Epoch: 25 | Loss: 0.17930432909471308\n",
            "Training Epoch: 25 | Loss: 0.17931076332294504\n",
            "Training Epoch: 25 | Loss: 0.17901938823390534\n",
            "Training Epoch: 25 | Loss: 0.17933988871376091\n",
            "Validation Loss: 0.16753017529845238\n",
            "Validation Loss: 0.19168283893625335\n",
            "Validation Loss: 0.18912111804241416\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 26 | Loss: 0.19842606037855148\n",
            "Training Epoch: 26 | Loss: 0.17232090076155002\n",
            "Training Epoch: 26 | Loss: 0.17404945317970877\n",
            "Training Epoch: 26 | Loss: 0.17182544406590072\n",
            "Training Epoch: 26 | Loss: 0.1745193297939928\n",
            "Training Epoch: 26 | Loss: 0.17556633071754565\n",
            "Training Epoch: 26 | Loss: 0.1767324035097775\n",
            "Training Epoch: 26 | Loss: 0.17759696574623932\n",
            "Training Epoch: 26 | Loss: 0.1773701577543859\n",
            "Validation Loss: 0.1697067767381668\n",
            "Validation Loss: 0.19083106877411357\n",
            "Validation Loss: 0.19283857457895778\n",
            "\n",
            "Training Epoch: 27 | Loss: 0.20691899210214615\n",
            "Training Epoch: 27 | Loss: 0.17567943114012774\n",
            "Training Epoch: 27 | Loss: 0.17492046841971615\n",
            "Training Epoch: 27 | Loss: 0.17283289590262593\n",
            "Training Epoch: 27 | Loss: 0.17360142392786848\n",
            "Training Epoch: 27 | Loss: 0.1753042704358831\n",
            "Training Epoch: 27 | Loss: 0.17597079508939445\n",
            "Training Epoch: 27 | Loss: 0.17577971016307933\n",
            "Training Epoch: 27 | Loss: 0.17524438374123527\n",
            "Validation Loss: 0.20292261242866516\n",
            "Validation Loss: 0.18492771926714052\n",
            "Validation Loss: 0.18554761032782383\n",
            "\n",
            "Training Epoch: 28 | Loss: 0.20249639078974724\n",
            "Training Epoch: 28 | Loss: 0.17074983258095414\n",
            "Training Epoch: 28 | Loss: 0.16849764063027664\n",
            "Training Epoch: 28 | Loss: 0.1701213106469706\n",
            "Training Epoch: 28 | Loss: 0.1700671935486526\n",
            "Training Epoch: 28 | Loss: 0.1699974855237051\n",
            "Training Epoch: 28 | Loss: 0.1698463109589977\n",
            "Training Epoch: 28 | Loss: 0.1700528596173148\n",
            "Training Epoch: 28 | Loss: 0.17064785336629115\n",
            "Validation Loss: 0.20899496227502823\n",
            "Validation Loss: 0.18283661828367131\n",
            "Validation Loss: 0.17996628589427738\n",
            "\n",
            "Training Epoch: 29 | Loss: 0.16723143681883812\n",
            "Training Epoch: 29 | Loss: 0.16498726173903389\n",
            "Training Epoch: 29 | Loss: 0.16189140676803404\n",
            "Training Epoch: 29 | Loss: 0.16562344011327754\n",
            "Training Epoch: 29 | Loss: 0.16539964532279908\n",
            "Training Epoch: 29 | Loss: 0.16627585826006658\n",
            "Training Epoch: 29 | Loss: 0.16672694578320532\n",
            "Training Epoch: 29 | Loss: 0.16684596137424\n",
            "Training Epoch: 29 | Loss: 0.1684172000101164\n",
            "Validation Loss: 0.22685733437538147\n",
            "Validation Loss: 0.1872056618443515\n",
            "Validation Loss: 0.18599905579614995\n",
            "\n",
            "Training Epoch: 30 | Loss: 0.16529813595116138\n",
            "Training Epoch: 30 | Loss: 0.15969805158490297\n",
            "Training Epoch: 30 | Loss: 0.15830484040971124\n",
            "Training Epoch: 30 | Loss: 0.1567927011632147\n",
            "Training Epoch: 30 | Loss: 0.15550987988924073\n",
            "Training Epoch: 30 | Loss: 0.15419294125588234\n",
            "Training Epoch: 30 | Loss: 0.15396508519164287\n",
            "Training Epoch: 30 | Loss: 0.15370267864779041\n",
            "Training Epoch: 30 | Loss: 0.15289312571673172\n",
            "Validation Loss: 0.1469468828290701\n",
            "Validation Loss: 0.1664571056492848\n",
            "Validation Loss: 0.1668232187990155\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 31 | Loss: 0.14909717440605164\n",
            "Training Epoch: 31 | Loss: 0.14830047251785747\n",
            "Training Epoch: 31 | Loss: 0.14714835821859426\n",
            "Training Epoch: 31 | Loss: 0.14696658467038526\n",
            "Training Epoch: 31 | Loss: 0.1473759038332023\n",
            "Training Epoch: 31 | Loss: 0.14681233402653904\n",
            "Training Epoch: 31 | Loss: 0.1472432877691757\n",
            "Training Epoch: 31 | Loss: 0.14672552769373937\n",
            "Training Epoch: 31 | Loss: 0.14602454004462775\n",
            "Validation Loss: 0.13192971237003803\n",
            "Validation Loss: 0.16626779473211506\n",
            "Validation Loss: 0.16314254069135556\n",
            "\n",
            "Training Epoch: 32 | Loss: 0.15958929061889648\n",
            "Training Epoch: 32 | Loss: 0.14507770151969526\n",
            "Training Epoch: 32 | Loss: 0.14501602531628527\n",
            "Training Epoch: 32 | Loss: 0.14574033380841891\n",
            "Training Epoch: 32 | Loss: 0.14597462175386097\n",
            "Training Epoch: 32 | Loss: 0.14530503421467905\n",
            "Training Epoch: 32 | Loss: 0.14497977259418557\n",
            "Training Epoch: 32 | Loss: 0.14500209721225113\n",
            "Training Epoch: 32 | Loss: 0.14483511763056287\n",
            "Validation Loss: 0.17656882293522358\n",
            "Validation Loss: 0.16422325930567366\n",
            "Validation Loss: 0.16268255127436337\n",
            "\n",
            "Training Epoch: 33 | Loss: 0.13364188931882381\n",
            "Training Epoch: 33 | Loss: 0.14090926241184962\n",
            "Training Epoch: 33 | Loss: 0.1416635098218436\n",
            "Training Epoch: 33 | Loss: 0.14276675977264577\n",
            "Training Epoch: 33 | Loss: 0.14268753822530594\n",
            "Training Epoch: 33 | Loss: 0.14264681345658625\n",
            "Training Epoch: 33 | Loss: 0.14288842276718078\n",
            "Training Epoch: 33 | Loss: 0.143205048892376\n",
            "Training Epoch: 33 | Loss: 0.14302573168647945\n",
            "Validation Loss: 0.12797581776976585\n",
            "Validation Loss: 0.16235577179543156\n",
            "Validation Loss: 0.16118916960905738\n",
            "\n",
            "Training Epoch: 34 | Loss: 0.1402034778147936\n",
            "Training Epoch: 34 | Loss: 0.14356202618357275\n",
            "Training Epoch: 34 | Loss: 0.1437118392650835\n",
            "Training Epoch: 34 | Loss: 0.14423270784070918\n",
            "Training Epoch: 34 | Loss: 0.14376815522503006\n",
            "Training Epoch: 34 | Loss: 0.14316016882881075\n",
            "Training Epoch: 34 | Loss: 0.14315142542489506\n",
            "Training Epoch: 34 | Loss: 0.14254543581145807\n",
            "Training Epoch: 34 | Loss: 0.14212952008919694\n",
            "Validation Loss: 0.20082200318574905\n",
            "Validation Loss: 0.15607534636660378\n",
            "Validation Loss: 0.16042465147613294\n",
            "\n",
            "Training Epoch: 35 | Loss: 0.13528640940785408\n",
            "Training Epoch: 35 | Loss: 0.14417017182095512\n",
            "Training Epoch: 35 | Loss: 0.1413894189540204\n",
            "Training Epoch: 35 | Loss: 0.1407534778922598\n",
            "Training Epoch: 35 | Loss: 0.1397215721808244\n",
            "Training Epoch: 35 | Loss: 0.13935297816627457\n",
            "Training Epoch: 35 | Loss: 0.13995435623337396\n",
            "Training Epoch: 35 | Loss: 0.14017588575138112\n",
            "Training Epoch: 35 | Loss: 0.14056728861135398\n",
            "Validation Loss: 0.14866485074162483\n",
            "Validation Loss: 0.1617887373275981\n",
            "Validation Loss: 0.16014964026931802\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 36 | Loss: 0.15206346102058887\n",
            "Training Epoch: 36 | Loss: 0.1370103362726398\n",
            "Training Epoch: 36 | Loss: 0.13863936688899253\n",
            "Training Epoch: 36 | Loss: 0.13884895035619554\n",
            "Training Epoch: 36 | Loss: 0.13928021289213116\n",
            "Training Epoch: 36 | Loss: 0.13960671930772786\n",
            "Training Epoch: 36 | Loss: 0.14015002151914002\n",
            "Training Epoch: 36 | Loss: 0.1397832968091584\n",
            "Training Epoch: 36 | Loss: 0.13992608518987318\n",
            "Validation Loss: 0.1887362077832222\n",
            "Validation Loss: 0.16176510182679585\n",
            "Validation Loss: 0.1594349093600848\n",
            "\n",
            "Training Epoch: 37 | Loss: 0.12116347625851631\n",
            "Training Epoch: 37 | Loss: 0.14185324240254588\n",
            "Training Epoch: 37 | Loss: 0.13897971340469015\n",
            "Training Epoch: 37 | Loss: 0.1390219624974205\n",
            "Training Epoch: 37 | Loss: 0.1385143224830435\n",
            "Training Epoch: 37 | Loss: 0.13875062918818432\n",
            "Training Epoch: 37 | Loss: 0.13926097735245022\n",
            "Training Epoch: 37 | Loss: 0.1390064984876211\n",
            "Training Epoch: 37 | Loss: 0.13889426892546566\n",
            "Validation Loss: 0.14551307447254658\n",
            "Validation Loss: 0.15676237156817524\n",
            "Validation Loss: 0.1586712980689249\n",
            "\n",
            "Training Epoch: 38 | Loss: 0.12586339749395847\n",
            "Training Epoch: 38 | Loss: 0.13451387627179376\n",
            "Training Epoch: 38 | Loss: 0.13464881319422922\n",
            "Training Epoch: 38 | Loss: 0.1353996942791333\n",
            "Training Epoch: 38 | Loss: 0.13649351933409\n",
            "Training Epoch: 38 | Loss: 0.13651473376297665\n",
            "Training Epoch: 38 | Loss: 0.1371574652460272\n",
            "Training Epoch: 38 | Loss: 0.13723955648724048\n",
            "Training Epoch: 38 | Loss: 0.13719154794359772\n",
            "Validation Loss: 0.13287467882037163\n",
            "Validation Loss: 0.15505467533216913\n",
            "Validation Loss: 0.15752896522426635\n",
            "\n",
            "Training Epoch: 39 | Loss: 0.13174331560730934\n",
            "Training Epoch: 39 | Loss: 0.1364056709969398\n",
            "Training Epoch: 39 | Loss: 0.1348766565591616\n",
            "Training Epoch: 39 | Loss: 0.13586195670934612\n",
            "Training Epoch: 39 | Loss: 0.136279559036963\n",
            "Training Epoch: 39 | Loss: 0.13624709930471673\n",
            "Training Epoch: 39 | Loss: 0.13635237778408357\n",
            "Training Epoch: 39 | Loss: 0.13632277147140295\n",
            "Training Epoch: 39 | Loss: 0.13636877567515465\n",
            "Validation Loss: 0.174721822142601\n",
            "Validation Loss: 0.1582106131193514\n",
            "Validation Loss: 0.15614745968869373\n",
            "\n",
            "Training Epoch: 40 | Loss: 0.12013669684529305\n",
            "Training Epoch: 40 | Loss: 0.13209482418871163\n",
            "Training Epoch: 40 | Loss: 0.1310471903963654\n",
            "Training Epoch: 40 | Loss: 0.13140457515075704\n",
            "Training Epoch: 40 | Loss: 0.13338948979606838\n",
            "Training Epoch: 40 | Loss: 0.13381757922080195\n",
            "Training Epoch: 40 | Loss: 0.13479780202062053\n",
            "Training Epoch: 40 | Loss: 0.13562371078466787\n",
            "Training Epoch: 40 | Loss: 0.13536351270910896\n",
            "Validation Loss: 0.11838603019714355\n",
            "Validation Loss: 0.1596005661689704\n",
            "Validation Loss: 0.15711133056945764\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 41 | Loss: 0.15194160863757133\n",
            "Training Epoch: 41 | Loss: 0.13543357050027882\n",
            "Training Epoch: 41 | Loss: 0.1346739257653406\n",
            "Training Epoch: 41 | Loss: 0.13473209078451526\n",
            "Training Epoch: 41 | Loss: 0.13497781919366553\n",
            "Training Epoch: 41 | Loss: 0.13496463281658297\n",
            "Training Epoch: 41 | Loss: 0.1357002083147053\n",
            "Training Epoch: 41 | Loss: 0.13586556979490924\n",
            "Training Epoch: 41 | Loss: 0.135459776118993\n",
            "Validation Loss: 0.17126787453889847\n",
            "Validation Loss: 0.15775642532043824\n",
            "Validation Loss: 0.15600556836681284\n",
            "\n",
            "Training Epoch: 42 | Loss: 0.14004615880548954\n",
            "Training Epoch: 42 | Loss: 0.13615811591285584\n",
            "Training Epoch: 42 | Loss: 0.13748657874837147\n",
            "Training Epoch: 42 | Loss: 0.13562041451145843\n",
            "Training Epoch: 42 | Loss: 0.13520506280067926\n",
            "Training Epoch: 42 | Loss: 0.13553559837189919\n",
            "Training Epoch: 42 | Loss: 0.13593661877948066\n",
            "Training Epoch: 42 | Loss: 0.13553828641389645\n",
            "Training Epoch: 42 | Loss: 0.13559328791302874\n",
            "Validation Loss: 0.11377754434943199\n",
            "Validation Loss: 0.15877565130185667\n",
            "Validation Loss: 0.1551260683611406\n",
            "\n",
            "Training Epoch: 43 | Loss: 0.1373677644878626\n",
            "Training Epoch: 43 | Loss: 0.13591830975532826\n",
            "Training Epoch: 43 | Loss: 0.1341278798442649\n",
            "Training Epoch: 43 | Loss: 0.13614904164859523\n",
            "Training Epoch: 43 | Loss: 0.13537751347235313\n",
            "Training Epoch: 43 | Loss: 0.13490857525046418\n",
            "Training Epoch: 43 | Loss: 0.13506662309907388\n",
            "Training Epoch: 43 | Loss: 0.13480673744456148\n",
            "Training Epoch: 43 | Loss: 0.13440374099648325\n",
            "Validation Loss: 0.18266897276043892\n",
            "Validation Loss: 0.15362528070696804\n",
            "Validation Loss: 0.15545869457873687\n",
            "\n",
            "Training Epoch: 44 | Loss: 0.1655827946960926\n",
            "Training Epoch: 44 | Loss: 0.13383885748721291\n",
            "Training Epoch: 44 | Loss: 0.1354383868651826\n",
            "Training Epoch: 44 | Loss: 0.135665253581374\n",
            "Training Epoch: 44 | Loss: 0.1353621544674076\n",
            "Training Epoch: 44 | Loss: 0.13529378123871402\n",
            "Training Epoch: 44 | Loss: 0.13447988849170445\n",
            "Training Epoch: 44 | Loss: 0.13455457718891442\n",
            "Training Epoch: 44 | Loss: 0.13433099319732814\n",
            "Validation Loss: 0.1199633814394474\n",
            "Validation Loss: 0.1522047169654086\n",
            "Validation Loss: 0.15508234385512212\n",
            "\n",
            "Training Epoch: 45 | Loss: 0.1016813088208437\n",
            "Training Epoch: 45 | Loss: 0.13548197761399322\n",
            "Training Epoch: 45 | Loss: 0.13246452249586582\n",
            "Training Epoch: 45 | Loss: 0.1331684324593342\n",
            "Training Epoch: 45 | Loss: 0.13304622235598781\n",
            "Training Epoch: 45 | Loss: 0.13363701260895905\n",
            "Training Epoch: 45 | Loss: 0.13372622288224253\n",
            "Training Epoch: 45 | Loss: 0.13349510378040874\n",
            "Training Epoch: 45 | Loss: 0.13325081580201079\n",
            "Validation Loss: 0.18766889162361622\n",
            "Validation Loss: 0.16213456544066124\n",
            "Validation Loss: 0.15595552954470637\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 46 | Loss: 0.1001485325396061\n",
            "Training Epoch: 46 | Loss: 0.1320281652272633\n",
            "Training Epoch: 46 | Loss: 0.13265821776950537\n",
            "Training Epoch: 46 | Loss: 0.13163342526758035\n",
            "Training Epoch: 46 | Loss: 0.1317113782383706\n",
            "Training Epoch: 46 | Loss: 0.132570140413241\n",
            "Training Epoch: 46 | Loss: 0.1321004900830285\n",
            "Training Epoch: 46 | Loss: 0.13238011580359155\n",
            "Training Epoch: 46 | Loss: 0.1327636790090937\n",
            "Validation Loss: 0.19173771515488625\n",
            "Validation Loss: 0.15458102435758797\n",
            "Validation Loss: 0.153134662981159\n",
            "\n",
            "Training Epoch: 47 | Loss: 0.15653258934617043\n",
            "Training Epoch: 47 | Loss: 0.1294304019504107\n",
            "Training Epoch: 47 | Loss: 0.13065201996719067\n",
            "Training Epoch: 47 | Loss: 0.13029246379704867\n",
            "Training Epoch: 47 | Loss: 0.13103944137738902\n",
            "Training Epoch: 47 | Loss: 0.13147360801756264\n",
            "Training Epoch: 47 | Loss: 0.13100280642149947\n",
            "Training Epoch: 47 | Loss: 0.1310941495211229\n",
            "Training Epoch: 47 | Loss: 0.13101237383990846\n",
            "Validation Loss: 0.13723058626055717\n",
            "Validation Loss: 0.15431909103610433\n",
            "Validation Loss: 0.1541624502255697\n",
            "\n",
            "Training Epoch: 48 | Loss: 0.15290119219571352\n",
            "Training Epoch: 48 | Loss: 0.12948833086002287\n",
            "Training Epoch: 48 | Loss: 0.13194830720289727\n",
            "Training Epoch: 48 | Loss: 0.13035199688778465\n",
            "Training Epoch: 48 | Loss: 0.13170468937861093\n",
            "Training Epoch: 48 | Loss: 0.13136387930275825\n",
            "Training Epoch: 48 | Loss: 0.13094461931864082\n",
            "Training Epoch: 48 | Loss: 0.13189542545111965\n",
            "Training Epoch: 48 | Loss: 0.13213370245743017\n",
            "Validation Loss: 0.15939001739025116\n",
            "Validation Loss: 0.15389733819641394\n",
            "Validation Loss: 0.15335870068056964\n",
            "\n",
            "Training Epoch: 49 | Loss: 0.13691089488565922\n",
            "Training Epoch: 49 | Loss: 0.13416821638693904\n",
            "Training Epoch: 49 | Loss: 0.13261513946235032\n",
            "Training Epoch: 49 | Loss: 0.13064403909929964\n",
            "Training Epoch: 49 | Loss: 0.13070844519799785\n",
            "Training Epoch: 49 | Loss: 0.13096214360752684\n",
            "Training Epoch: 49 | Loss: 0.13082386001467258\n",
            "Training Epoch: 49 | Loss: 0.13038697931915522\n",
            "Training Epoch: 49 | Loss: 0.13071093950932205\n",
            "Validation Loss: 0.178191052749753\n",
            "Validation Loss: 0.1522629468766327\n",
            "Validation Loss: 0.15497970611172085\n",
            "\n",
            "Training Epoch: 50 | Loss: 0.11077447608113289\n",
            "Training Epoch: 50 | Loss: 0.13017049287953\n",
            "Training Epoch: 50 | Loss: 0.13063099772775943\n",
            "Training Epoch: 50 | Loss: 0.13107738133049585\n",
            "Training Epoch: 50 | Loss: 0.13135379415813675\n",
            "Training Epoch: 50 | Loss: 0.13047613693352886\n",
            "Training Epoch: 50 | Loss: 0.1303818926103599\n",
            "Training Epoch: 50 | Loss: 0.13002324445882596\n",
            "Training Epoch: 50 | Loss: 0.12991924072091424\n",
            "Validation Loss: 0.15091070346534252\n",
            "Validation Loss: 0.15607136130646462\n",
            "Validation Loss: 0.15258882127804171\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 51 | Loss: 0.14594712108373642\n",
            "Training Epoch: 51 | Loss: 0.1248013979091429\n",
            "Training Epoch: 51 | Loss: 0.12692755241808829\n",
            "Training Epoch: 51 | Loss: 0.12831652512699612\n",
            "Training Epoch: 51 | Loss: 0.1287766135300038\n",
            "Training Epoch: 51 | Loss: 0.1293556450098426\n",
            "Training Epoch: 51 | Loss: 0.12954872124169808\n",
            "Training Epoch: 51 | Loss: 0.12916854118925655\n",
            "Training Epoch: 51 | Loss: 0.12901298613165424\n",
            "Validation Loss: 0.12869765795767307\n",
            "Validation Loss: 0.1487093988695357\n",
            "Validation Loss: 0.15183630250209007\n",
            "\n",
            "Training Epoch: 52 | Loss: 0.1026082169264555\n",
            "Training Epoch: 52 | Loss: 0.1277203207356062\n",
            "Training Epoch: 52 | Loss: 0.12851646512431736\n",
            "Training Epoch: 52 | Loss: 0.12691421936145456\n",
            "Training Epoch: 52 | Loss: 0.12800767434821314\n",
            "Training Epoch: 52 | Loss: 0.12778840859619503\n",
            "Training Epoch: 52 | Loss: 0.12744722976047118\n",
            "Training Epoch: 52 | Loss: 0.12811283808807375\n",
            "Training Epoch: 52 | Loss: 0.12794103360875567\n",
            "Validation Loss: 0.17501967959105968\n",
            "Validation Loss: 0.14930496980795765\n",
            "Validation Loss: 0.1512175345079816\n",
            "\n",
            "Training Epoch: 53 | Loss: 0.12318732403218746\n",
            "Training Epoch: 53 | Loss: 0.12571313781755986\n",
            "Training Epoch: 53 | Loss: 0.12531027791382215\n",
            "Training Epoch: 53 | Loss: 0.1280118446159749\n",
            "Training Epoch: 53 | Loss: 0.12885183682827803\n",
            "Training Epoch: 53 | Loss: 0.12872096125088528\n",
            "Training Epoch: 53 | Loss: 0.1282168173908369\n",
            "Training Epoch: 53 | Loss: 0.1281417754562452\n",
            "Training Epoch: 53 | Loss: 0.12845327450106672\n",
            "Validation Loss: 0.1680463943630457\n",
            "Validation Loss: 0.1487993952993414\n",
            "Validation Loss: 0.15177887790739092\n",
            "\n",
            "Training Epoch: 54 | Loss: 0.157408706843853\n",
            "Training Epoch: 54 | Loss: 0.12885089158679885\n",
            "Training Epoch: 54 | Loss: 0.1292324834758073\n",
            "Training Epoch: 54 | Loss: 0.1292016566406156\n",
            "Training Epoch: 54 | Loss: 0.12911464878011894\n",
            "Training Epoch: 54 | Loss: 0.12846023005483573\n",
            "Training Epoch: 54 | Loss: 0.1282795159333694\n",
            "Training Epoch: 54 | Loss: 0.12809466579185\n",
            "Training Epoch: 54 | Loss: 0.1279968148801992\n",
            "Validation Loss: 0.1426497958600521\n",
            "Validation Loss: 0.15228406022680868\n",
            "Validation Loss: 0.15141499909892012\n",
            "\n",
            "Training Epoch: 55 | Loss: 0.12593234796077013\n",
            "Training Epoch: 55 | Loss: 0.12468170839371068\n",
            "Training Epoch: 55 | Loss: 0.12628654101446493\n",
            "Training Epoch: 55 | Loss: 0.1270362139954866\n",
            "Training Epoch: 55 | Loss: 0.1265590923148535\n",
            "Training Epoch: 55 | Loss: 0.1272185357369319\n",
            "Training Epoch: 55 | Loss: 0.12739625376666247\n",
            "Training Epoch: 55 | Loss: 0.12763433552050854\n",
            "Training Epoch: 55 | Loss: 0.12793645397707654\n",
            "Validation Loss: 0.13698064535856247\n",
            "Validation Loss: 0.14954291616962984\n",
            "Validation Loss: 0.15078720647322746\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 56 | Loss: 0.1395331174135208\n",
            "Training Epoch: 56 | Loss: 0.12987285797106157\n",
            "Training Epoch: 56 | Loss: 0.1290082192738814\n",
            "Training Epoch: 56 | Loss: 0.13017010475753302\n",
            "Training Epoch: 56 | Loss: 0.12929143957872677\n",
            "Training Epoch: 56 | Loss: 0.12932097234348663\n",
            "Training Epoch: 56 | Loss: 0.12840478749598644\n",
            "Training Epoch: 56 | Loss: 0.12794643921044238\n",
            "Training Epoch: 56 | Loss: 0.12787054020030128\n",
            "Validation Loss: 0.10640829429030418\n",
            "Validation Loss: 0.1502487006606442\n",
            "Validation Loss: 0.1526404997259749\n",
            "\n",
            "Training Epoch: 57 | Loss: 0.1267650742083788\n",
            "Training Epoch: 57 | Loss: 0.12752719440193164\n",
            "Training Epoch: 57 | Loss: 0.12863848172359874\n",
            "Training Epoch: 57 | Loss: 0.12965188201275527\n",
            "Training Epoch: 57 | Loss: 0.1293217530276654\n",
            "Training Epoch: 57 | Loss: 0.12848410645533287\n",
            "Training Epoch: 57 | Loss: 0.128566529316682\n",
            "Training Epoch: 57 | Loss: 0.128588631883338\n",
            "Training Epoch: 57 | Loss: 0.12833783519890313\n",
            "Validation Loss: 0.17916010320186615\n",
            "Validation Loss: 0.14821014402083832\n",
            "Validation Loss: 0.14964410310045848\n",
            "\n",
            "Training Epoch: 58 | Loss: 0.15045594237744808\n",
            "Training Epoch: 58 | Loss: 0.12798642137521268\n",
            "Training Epoch: 58 | Loss: 0.12707230816850096\n",
            "Training Epoch: 58 | Loss: 0.1268813316339002\n",
            "Training Epoch: 58 | Loss: 0.12773585151486155\n",
            "Training Epoch: 58 | Loss: 0.12731611599193926\n",
            "Training Epoch: 58 | Loss: 0.12746794341820078\n",
            "Training Epoch: 58 | Loss: 0.12768448533134244\n",
            "Training Epoch: 58 | Loss: 0.1281722142726192\n",
            "Validation Loss: 0.13749243319034576\n",
            "Validation Loss: 0.1494210528597088\n",
            "Validation Loss: 0.14962688252904374\n",
            "\n",
            "Training Epoch: 59 | Loss: 0.11740010604262352\n",
            "Training Epoch: 59 | Loss: 0.1296359763508386\n",
            "Training Epoch: 59 | Loss: 0.12842194597463852\n",
            "Training Epoch: 59 | Loss: 0.12853432792638028\n",
            "Training Epoch: 59 | Loss: 0.1282140888677831\n",
            "Training Epoch: 59 | Loss: 0.12872188173963162\n",
            "Training Epoch: 59 | Loss: 0.12842111922676572\n",
            "Training Epoch: 59 | Loss: 0.12838443320712206\n",
            "Training Epoch: 59 | Loss: 0.12833185713987022\n",
            "Validation Loss: 0.07724031619727612\n",
            "Validation Loss: 0.15177084505558014\n",
            "Validation Loss: 0.15008846062249434\n",
            "\n",
            "Training Epoch: 60 | Loss: 0.12128831539303064\n",
            "Training Epoch: 60 | Loss: 0.13097195287631586\n",
            "Training Epoch: 60 | Loss: 0.12661357526088235\n",
            "Training Epoch: 60 | Loss: 0.12686910008101962\n",
            "Training Epoch: 60 | Loss: 0.12787191438583884\n",
            "Training Epoch: 60 | Loss: 0.12746422398582608\n",
            "Training Epoch: 60 | Loss: 0.12757100009618205\n",
            "Training Epoch: 60 | Loss: 0.1279042687243151\n",
            "Training Epoch: 60 | Loss: 0.1278674630552865\n",
            "Validation Loss: 0.16210730001330376\n",
            "Validation Loss: 0.14910003696891047\n",
            "Validation Loss: 0.15146952061751737\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 61 | Loss: 0.1009722352027893\n",
            "Training Epoch: 61 | Loss: 0.12856135864991067\n",
            "Training Epoch: 61 | Loss: 0.1271134881506586\n",
            "Training Epoch: 61 | Loss: 0.12666137749793324\n",
            "Training Epoch: 61 | Loss: 0.1271813815515349\n",
            "Training Epoch: 61 | Loss: 0.12672419499182058\n",
            "Training Epoch: 61 | Loss: 0.12749122952834738\n",
            "Training Epoch: 61 | Loss: 0.1270619197403175\n",
            "Training Epoch: 61 | Loss: 0.1270171895410433\n",
            "Validation Loss: 0.22367330081760883\n",
            "Validation Loss: 0.1555609964571967\n",
            "Validation Loss: 0.1511726364845512\n",
            "\n",
            "Training Epoch: 62 | Loss: 0.0968262804672122\n",
            "Training Epoch: 62 | Loss: 0.1288273635108282\n",
            "Training Epoch: 62 | Loss: 0.13015623500950596\n",
            "Training Epoch: 62 | Loss: 0.12975603003252878\n",
            "Training Epoch: 62 | Loss: 0.1282932187507203\n",
            "Training Epoch: 62 | Loss: 0.12777905889643643\n",
            "Training Epoch: 62 | Loss: 0.12781590865010678\n",
            "Training Epoch: 62 | Loss: 0.12710252948392306\n",
            "Training Epoch: 62 | Loss: 0.1273035613962969\n",
            "Validation Loss: 0.20092477276921272\n",
            "Validation Loss: 0.14634622573520584\n",
            "Validation Loss: 0.14958436591133698\n",
            "\n",
            "Training Epoch: 63 | Loss: 0.1328619448468089\n",
            "Training Epoch: 63 | Loss: 0.1250622231734566\n",
            "Training Epoch: 63 | Loss: 0.1278040915153068\n",
            "Training Epoch: 63 | Loss: 0.12809993361008426\n",
            "Training Epoch: 63 | Loss: 0.1271036160282995\n",
            "Training Epoch: 63 | Loss: 0.1269576003023994\n",
            "Training Epoch: 63 | Loss: 0.12764415520238698\n",
            "Training Epoch: 63 | Loss: 0.12756464762291664\n",
            "Training Epoch: 63 | Loss: 0.1277939265014862\n",
            "Validation Loss: 0.11774739250540733\n",
            "Validation Loss: 0.14967101865844562\n",
            "Validation Loss: 0.14908855366265744\n",
            "\n",
            "Training Epoch: 64 | Loss: 0.13459014892578125\n",
            "Training Epoch: 64 | Loss: 0.12855691368700845\n",
            "Training Epoch: 64 | Loss: 0.12647176754489467\n",
            "Training Epoch: 64 | Loss: 0.1273612398313872\n",
            "Training Epoch: 64 | Loss: 0.12848778431561597\n",
            "Training Epoch: 64 | Loss: 0.1275263849976742\n",
            "Training Epoch: 64 | Loss: 0.12700846751907008\n",
            "Training Epoch: 64 | Loss: 0.12724268864361987\n",
            "Training Epoch: 64 | Loss: 0.12745972435640057\n",
            "Validation Loss: 0.12036133836954832\n",
            "Validation Loss: 0.15092666026684318\n",
            "Validation Loss: 0.15020315876851478\n",
            "\n",
            "Training Epoch: 65 | Loss: 0.12018739990890026\n",
            "Training Epoch: 65 | Loss: 0.11983188196076172\n",
            "Training Epoch: 65 | Loss: 0.12330057179851837\n",
            "Training Epoch: 65 | Loss: 0.12433950053342306\n",
            "Training Epoch: 65 | Loss: 0.1251247344698822\n",
            "Training Epoch: 65 | Loss: 0.12581501343378615\n",
            "Training Epoch: 65 | Loss: 0.1263137504128358\n",
            "Training Epoch: 65 | Loss: 0.12679954606989205\n",
            "Training Epoch: 65 | Loss: 0.12687477676288578\n",
            "Validation Loss: 0.1277560368180275\n",
            "Validation Loss: 0.1495156640758609\n",
            "Validation Loss: 0.15101347709843768\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 66 | Loss: 0.12618125975131989\n",
            "Training Epoch: 66 | Loss: 0.12914780272853257\n",
            "Training Epoch: 66 | Loss: 0.12789692004113945\n",
            "Training Epoch: 66 | Loss: 0.12956510222458167\n",
            "Training Epoch: 66 | Loss: 0.128554311645986\n",
            "Training Epoch: 66 | Loss: 0.1279715334665692\n",
            "Training Epoch: 66 | Loss: 0.12754602825575928\n",
            "Training Epoch: 66 | Loss: 0.12753502789821033\n",
            "Training Epoch: 66 | Loss: 0.1278280612527766\n",
            "Validation Loss: 0.13987663388252258\n",
            "Validation Loss: 0.1486371211744476\n",
            "Validation Loss: 0.15112651234483393\n",
            "\n",
            "Training Epoch: 67 | Loss: 0.14122574217617512\n",
            "Training Epoch: 67 | Loss: 0.12878868594249288\n",
            "Training Epoch: 67 | Loss: 0.12703893381397968\n",
            "Training Epoch: 67 | Loss: 0.1269984537135327\n",
            "Training Epoch: 67 | Loss: 0.12724814419065628\n",
            "Training Epoch: 67 | Loss: 0.12717476285373172\n",
            "Training Epoch: 67 | Loss: 0.12725587485872594\n",
            "Training Epoch: 67 | Loss: 0.1274904737537315\n",
            "Training Epoch: 67 | Loss: 0.1274024349737322\n",
            "Validation Loss: 0.15735146217048168\n",
            "Validation Loss: 0.14830515268120434\n",
            "Validation Loss: 0.15053790155798197\n",
            "\n",
            "Training Epoch: 68 | Loss: 0.12716376222670078\n",
            "Training Epoch: 68 | Loss: 0.12736591559346064\n",
            "Training Epoch: 68 | Loss: 0.12865772544040313\n",
            "Training Epoch: 68 | Loss: 0.12763967021581937\n",
            "Training Epoch: 68 | Loss: 0.12744149503685961\n",
            "Training Epoch: 68 | Loss: 0.12861720120493406\n",
            "Training Epoch: 68 | Loss: 0.12832985722774426\n",
            "Training Epoch: 68 | Loss: 0.12743003510959053\n",
            "Training Epoch: 68 | Loss: 0.12805094794903912\n",
            "Validation Loss: 0.10288998484611511\n",
            "Validation Loss: 0.15054282096979937\n",
            "Validation Loss: 0.1506179288187208\n",
            "\n",
            "Training Epoch: 69 | Loss: 0.11506218649446964\n",
            "Training Epoch: 69 | Loss: 0.1267571442133498\n",
            "Training Epoch: 69 | Loss: 0.12710285035948923\n",
            "Training Epoch: 69 | Loss: 0.1268626664072102\n",
            "Training Epoch: 69 | Loss: 0.12648816769269117\n",
            "Training Epoch: 69 | Loss: 0.12653037669609288\n",
            "Training Epoch: 69 | Loss: 0.12622162846913454\n",
            "Training Epoch: 69 | Loss: 0.12664508748900533\n",
            "Training Epoch: 69 | Loss: 0.1264177451454354\n",
            "Validation Loss: 0.19796617329120636\n",
            "Validation Loss: 0.15189363265244088\n",
            "Validation Loss: 0.1508083722290041\n",
            "\n",
            "Training Epoch: 70 | Loss: 0.10773616470396519\n",
            "Training Epoch: 70 | Loss: 0.128474870770432\n",
            "Training Epoch: 70 | Loss: 0.12983429949934505\n",
            "Training Epoch: 70 | Loss: 0.12926607564295892\n",
            "Training Epoch: 70 | Loss: 0.12761061936222073\n",
            "Training Epoch: 70 | Loss: 0.12637498842537434\n",
            "Training Epoch: 70 | Loss: 0.12598427194178402\n",
            "Training Epoch: 70 | Loss: 0.12655419371517299\n",
            "Training Epoch: 70 | Loss: 0.12678521286961292\n",
            "Validation Loss: 0.091615816578269\n",
            "Validation Loss: 0.1482983855069569\n",
            "Validation Loss: 0.14903987743961278\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 71 | Loss: 0.119870625436306\n",
            "Training Epoch: 71 | Loss: 0.1270487995560069\n",
            "Training Epoch: 71 | Loss: 0.12780435881878607\n",
            "Training Epoch: 71 | Loss: 0.12779726667416302\n",
            "Training Epoch: 71 | Loss: 0.1272397001315735\n",
            "Training Epoch: 71 | Loss: 0.12743296084680153\n",
            "Training Epoch: 71 | Loss: 0.1277789340149952\n",
            "Training Epoch: 71 | Loss: 0.12802251811589668\n",
            "Training Epoch: 71 | Loss: 0.12723320328947924\n",
            "Validation Loss: 0.13389458786696196\n",
            "Validation Loss: 0.15103914619119155\n",
            "Validation Loss: 0.1495876892455923\n",
            "\n",
            "Training Epoch: 72 | Loss: 0.11554756946861744\n",
            "Training Epoch: 72 | Loss: 0.12494182554396367\n",
            "Training Epoch: 72 | Loss: 0.12652485333475752\n",
            "Training Epoch: 72 | Loss: 0.126107120787434\n",
            "Training Epoch: 72 | Loss: 0.12503956042304448\n",
            "Training Epoch: 72 | Loss: 0.1259175871114085\n",
            "Training Epoch: 72 | Loss: 0.1261279210014371\n",
            "Training Epoch: 72 | Loss: 0.12645816733228965\n",
            "Training Epoch: 72 | Loss: 0.1266008523094018\n",
            "Validation Loss: 0.13717797584831715\n",
            "Validation Loss: 0.15016679458393908\n",
            "Validation Loss: 0.15040686880400525\n",
            "\n",
            "Training Epoch: 73 | Loss: 0.11150839924812317\n",
            "Training Epoch: 73 | Loss: 0.12622973509132862\n",
            "Training Epoch: 73 | Loss: 0.12582930520788502\n",
            "Training Epoch: 73 | Loss: 0.12695011777797113\n",
            "Training Epoch: 73 | Loss: 0.12597413165053525\n",
            "Training Epoch: 73 | Loss: 0.12630729651089737\n",
            "Training Epoch: 73 | Loss: 0.12559368779197855\n",
            "Training Epoch: 73 | Loss: 0.1260849476968617\n",
            "Training Epoch: 73 | Loss: 0.12601767399100663\n",
            "Validation Loss: 0.16582746803760529\n",
            "Validation Loss: 0.15618922199021176\n",
            "Validation Loss: 0.15036561577789373\n",
            "\n",
            "Training Epoch: 74 | Loss: 0.11187710613012314\n",
            "Training Epoch: 74 | Loss: 0.12561039500270443\n",
            "Training Epoch: 74 | Loss: 0.12481983703797433\n",
            "Training Epoch: 74 | Loss: 0.12480250020137063\n",
            "Training Epoch: 74 | Loss: 0.1253579344990619\n",
            "Training Epoch: 74 | Loss: 0.12535231386101295\n",
            "Training Epoch: 74 | Loss: 0.12617940784504827\n",
            "Training Epoch: 74 | Loss: 0.12600648369289372\n",
            "Training Epoch: 74 | Loss: 0.12668255545412854\n",
            "Validation Loss: 0.1768135130405426\n",
            "Validation Loss: 0.15619683119993988\n",
            "Validation Loss: 0.15040244555462207\n",
            "\n",
            "Training Epoch: 75 | Loss: 0.14798284880816936\n",
            "Training Epoch: 75 | Loss: 0.13024941934180437\n",
            "Training Epoch: 75 | Loss: 0.12837038023063718\n",
            "Training Epoch: 75 | Loss: 0.12659150076584513\n",
            "Training Epoch: 75 | Loss: 0.12679687505959536\n",
            "Training Epoch: 75 | Loss: 0.1277771553149198\n",
            "Training Epoch: 75 | Loss: 0.1268029183950492\n",
            "Training Epoch: 75 | Loss: 0.12761387789818576\n",
            "Training Epoch: 75 | Loss: 0.12695913468679407\n",
            "Validation Loss: 0.19182095490396023\n",
            "Validation Loss: 0.14630295800063575\n",
            "Validation Loss: 0.15051557115784184\n",
            "\n",
            "==> Save checkpoint ...\n",
            "Training Epoch: 76 | Loss: 0.1368694957345724\n",
            "Training Epoch: 76 | Loss: 0.1234776968330071\n",
            "Training Epoch: 76 | Loss: 0.12452868155588336\n",
            "Training Epoch: 76 | Loss: 0.12583596761991267\n",
            "Training Epoch: 76 | Loss: 0.12565489540787\n",
            "Training Epoch: 76 | Loss: 0.12615287114128085\n",
            "Training Epoch: 76 | Loss: 0.1266467268137066\n",
            "Training Epoch: 76 | Loss: 0.12699277936806438\n",
            "Training Epoch: 76 | Loss: 0.12717556874134048\n",
            "Validation Loss: 0.12894094735383987\n",
            "Validation Loss: 0.1513928957375707\n",
            "Validation Loss: 0.15112463699941017\n",
            "\n",
            "Training Epoch: 77 | Loss: 0.17138062790036201\n",
            "Training Epoch: 77 | Loss: 0.13003266067013586\n",
            "Training Epoch: 77 | Loss: 0.12741345893220968\n",
            "Training Epoch: 77 | Loss: 0.12730262023505085\n",
            "Training Epoch: 77 | Loss: 0.1280169491528387\n",
            "Training Epoch: 77 | Loss: 0.12799757826392968\n",
            "Training Epoch: 77 | Loss: 0.12796731542148154\n",
            "Training Epoch: 77 | Loss: 0.12752940581803252\n",
            "Training Epoch: 77 | Loss: 0.12767637710606727\n",
            "Validation Loss: 0.13769418373703957\n",
            "Validation Loss: 0.15339837622561373\n",
            "Validation Loss: 0.15109787721057139\n",
            "\n",
            "Training Epoch: 78 | Loss: 0.10369015857577324\n",
            "Training Epoch: 78 | Loss: 0.12977484022128846\n",
            "Training Epoch: 78 | Loss: 0.1274376038383155\n",
            "Training Epoch: 78 | Loss: 0.12747820224334483\n",
            "Training Epoch: 78 | Loss: 0.12801938025661724\n",
            "Training Epoch: 78 | Loss: 0.1273163602363965\n",
            "Training Epoch: 78 | Loss: 0.12742579778030316\n",
            "Training Epoch: 78 | Loss: 0.12668280426835304\n",
            "Training Epoch: 78 | Loss: 0.12663664867678254\n",
            "Validation Loss: 0.1354389674961567\n",
            "Validation Loss: 0.1471553150435338\n",
            "Validation Loss: 0.14979947082216466\n",
            "\n",
            "Training Epoch: 79 | Loss: 0.1179228201508522\n",
            "Training Epoch: 79 | Loss: 0.12421518261774932\n",
            "Training Epoch: 79 | Loss: 0.12445577786336491\n",
            "Training Epoch: 79 | Loss: 0.1245679868651288\n",
            "Training Epoch: 79 | Loss: 0.12524385885579367\n",
            "Training Epoch: 79 | Loss: 0.12624260584569114\n",
            "Training Epoch: 79 | Loss: 0.126193331149306\n",
            "Training Epoch: 79 | Loss: 0.12636925383921335\n",
            "Training Epoch: 79 | Loss: 0.12646136710655118\n",
            "Validation Loss: 0.12799897138029337\n",
            "Validation Loss: 0.14879305375795257\n",
            "Validation Loss: 0.14951159322949414\n",
            "\n",
            "==> Save checkpoint ...\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}